/**
 * DSM - Distributed Search Manager
 * Developed by Milspec Research International Pty Ltd
 * $Author: vhobbs $
 * $Revision: 1.12 $
 * $Date: 2007/01/03 23:03:37 $
 * (c)Copyright 2004
 * education.au limited
 * DEST
 * All rights reserved.
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 * Redistributions of source code must retain the above copyright notice, this
 * list of conditions and the following disclaimer.
 * Redistributions in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer in the documentation
 * and/or other materials provided with the distribution.
 * Neither the names education.au limited, DEST nor the names of its
 * contributors may be used to endorse or promote products derived from this
 * software without specific prior written permission.
 * To the extent permitted by law, the copyright owners of this software and
 * its contributors:
 * (i) exclude all warranties in relation to the software; and
 * (ii) exclude liability for all losses, costs, expenses and damages arising
 * in any way from the use of the software whether arising from or in
 * relation to breach of contract, negligence or any other tort, in equity
 * or otherwise. If the software is in breach of a warranty which is
 * implied by law, the copyright owners and contributors liability is
 * limited to the replacement of the software.
 *
 * @author	gsingh
 */

package au.edu.educationau.opensource.dsm.worker;

// standard java stuff
import java.io.Reader;
import java.util.List;

import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import org.xml.sax.XMLReader;
import org.xml.sax.helpers.DefaultHandler;
import org.xml.sax.helpers.XMLReaderFactory;

import au.edu.educationau.opensource.dsm.obj.SearchCriteria;
import au.edu.educationau.opensource.dsm.obj.SearchResult;
import au.edu.educationau.opensource.dsm.util.Flog;

/**
 * Contains methods to process XML, calculate hit frequencies and relevances for
 * result sets.
 */

public final class ResultSetUnifier {
	/*
	 * static { // Use the Piccolo SAX Parser. Fastest - apparently.
	 * System.setProperty( "org.xml.sax.driver", "com.bluecast.xml.Piccolo");
	 * System.setProperty( "javax.xml.parsers.SAXParserFactory",
	 * "com.bluecast.xml.JAXPSAXParserFactory" ); }
	 */
	/** The upper limit for a calculated relevance to reduce noise contamination */
	public static final float RELEVANCE_UPPER_LIMIT = 1.00f;
	/** The lower limit for a calculated relevance to reduce noise contamination */
	public static final float RELEVANCE_LOWER_LIMIT = 0.10f;
	/** Title hit frequency weight */
	public static final float HIT_FREQEUNCY_TITLE = 2.0f;
	/** Description hit frequency weight */
	public static final float HIT_FREQEUNCY_DESCRIPTION = 1.0f;

	/**
	 * Uses a contenthandler to parse and build a result list. An InputSource
	 * and DefaultHandler implemented object are passed in and then event based
	 * parsing updates the DefaultHandler class. They can be found in
	 * au.edu.educationau.dsm.adapters.contenthandler
	 * 
	 * @param source
	 *            Raw InputSource probably from a PipedOutputStream
	 * @param handler
	 *            customised handler for each sax result
	 * 
	 * @return DefaultHandler handler to return with results
	 */
	public static DefaultHandler processXML(InputSource source, DefaultHandler handler) {
		XMLReader xr = null;
		try {
			xr = XMLReaderFactory.createXMLReader();
			xr.setContentHandler(handler);

			try {
				xr.setFeature("http://xml.org/sax/features/namespaces", true);
				xr.setFeature("http://xml.org/sax/features/namespace-prefixes", false);
			} catch (SAXException e) {
				throw new Exception(e);
			}

		} catch (Exception ex) {
			Flog.error(classStr, "Error Building Parser", ex);
		}

		try {
			xr.parse(source);
		} catch (Exception t) {
			Flog.error(classStr, "Error Parsing Content", t);
		}

		return handler;
	} // processXML

	/**
	 * Uses a contenthandler to parse and build a result list. An InputSource
	 * and DefaultHandler implemented object are passed in and then event based
	 * parsing updates the DefaultHandler class. They can be found in
	 * au.edu.educationau.dsm.adapters.contenthandler
	 * 
	 * @param source
	 *            Raw InputSource probably from a PipedOutputStream
	 * @param handler
	 *            customised handler for each sax result
	 * 
	 * @return DefaultHandler handler to return with results
	 */
	public static DefaultHandler processXHTML(InputSource source, DefaultHandler handler) {
		XMLReader xr = null;
		try {
			// use the tagsoup parser, tolerant of dodgy XHTML.
			xr = XMLReaderFactory.createXMLReader("org.ccil.cowan.tagsoup.Parser");
			xr.setContentHandler(handler);

			try {
				xr.setFeature("http://xml.org/sax/features/namespaces", true);
				xr.setFeature("http://xml.org/sax/features/namespace-prefixes", false);
			} catch (SAXException e) {
				throw new Exception(e);
			}

		} catch (Exception ex) {
			Flog.error(classStr, "Error Building Parser", ex);
		}

		try {
			xr.parse(source);
		} catch (Exception t) {
			Flog.error(classStr, "Error Parsing Content", t);
		}

		return handler;
	} // processXHTML

	/**
	 * Uses a contenthandler to parse and build a result list. A java.io.Reader
	 * and DefaultHandler implemented object are passed in and then event based
	 * parsing updates the DefaultHandler class. They can be found in
	 * au.edu.educationau.dsm.adapters.contenthandler
	 * 
	 * @param reader
	 *            BufferedReader object that contains the stream of data
	 * @param handler
	 *            customised handler for each sax result
	 * @param encoding
	 *            the character encoding
	 * 
	 * @return DefaultHandler handler to return with results
	 */
	public static DefaultHandler processXHTML(Reader reader, DefaultHandler handler) {
		return processXHTML(reader, handler, null);
	}

	/**
	 * Uses a contenthandler to parse and build a result list. A java.io.Reader
	 * and DefaultHandler implemented object are passed in and then event based
	 * parsing updates the DefaultHandler class. They can be found in
	 * au.edu.educationau.dsm.adapters.contenthandler
	 * 
	 * @param reader
	 *            BufferedReader object that contains the stream of data
	 * @param handler
	 *            customised handler for each sax result
	 * @param encoding
	 *            the character encoding
	 * 
	 * @return DefaultHandler handler to return with results
	 */
	public static DefaultHandler processXML(Reader reader, DefaultHandler handler) {
		return processXML(reader, handler, null);
	}

	/**
	 * Uses a contenthandler to parse and build a result list. A java.io.Reader
	 * and DefaultHandler implemented object are passed in and then event based
	 * parsing updates the DefaultHandler class. They can be found in
	 * au.edu.educationau.dsm.adapters.contenthandler
	 * 
	 * @param reader
	 *            BufferedReader object that contains the stream of data
	 * @param handler
	 *            customised handler for each sax result
	 * @param encoding
	 *            the character encoding
	 * 
	 * @return DefaultHandler handler to return with results
	 */
	public static DefaultHandler processXML(Reader reader, DefaultHandler handler, String encoding) {
		XMLReader xr = null;
		try {
			xr = XMLReaderFactory.createXMLReader();
			xr.setContentHandler(handler);

			try {
				xr.setFeature("http://xml.org/sax/features/namespaces", true);
				xr.setFeature("http://xml.org/sax/features/namespace-prefixes", false);
			} catch (SAXException e) {
				throw new Exception(e);
			}

		} catch (Exception ex) {
			Flog.error(classStr, "Error Building Parser", ex);
		}

		try {
			if (encoding == null || encoding.length() <= 0) {
				xr.parse(new InputSource(reader));
			} else {
				InputSource src = new InputSource(reader);
				src.setEncoding(encoding);
				xr.parse(src);
			}
		} catch (Exception t) {
			Flog.error(classStr, "Error Parsing Content", t);
		}

		return handler;
	} // processXML

	/**
	 * Uses a contenthandler to parse and build a result list. A java.io.Reader
	 * and DefaultHandler implemented object are passed in and then event based
	 * parsing updates the DefaultHandler class. They can be found in
	 * au.edu.educationau.dsm.adapters.contenthandler
	 * 
	 * @param reader
	 *            BufferedReader object that contains the stream of data
	 * @param handler
	 *            customised handler for each sax result
	 * @param encoding
	 *            the character encoding
	 * 
	 * @return DefaultHandler handler to return with results
	 */
	public static DefaultHandler processXHTML(Reader reader, DefaultHandler handler, String encoding) {
		XMLReader xr = null;
		try {
			xr = XMLReaderFactory.createXMLReader("org.ccil.cowan.tagsoup.Parser");
			xr.setContentHandler(handler);

			try {
				xr.setFeature("http://xml.org/sax/features/namespaces", true);
				xr.setFeature("http://xml.org/sax/features/namespace-prefixes", false);
			} catch (SAXException e) {
				throw new Exception(e);
			}

		} catch (Exception ex) {
			Flog.error(classStr, "Error Building Parser", ex);
		}

		try {
			if (encoding == null || encoding.length() <= 0) {
				xr.parse(new InputSource(reader));
			} else {
				InputSource src = new InputSource(reader);
				src.setEncoding(encoding);
				xr.parse(src);
			}
		} catch (Exception t) {
			Flog.error(classStr, "Error Parsing Content", t);
		}

		return handler;
	} // processXML

	/**
	 * Calculates the hit frequency for the result. Is a wrapper for
	 * getHitFrequency. See calculateFrequency(String keyword, String txt)
	 * 
	 * @param keywords
	 *            array of String words to search for
	 * @param title
	 *            the String text to search in
	 * @param description
	 *            the String text to search in
	 * @param constraint
	 *            the keyword constraint determines how the keywords are
	 *            searched for
	 * @param isCaseSensitive
	 *            the case sensitivity determines whether case is preserved in
	 *            the searching
	 * 
	 * @return float freq
	 */
	public static float getHitFrequency(String[] keywords, String title, String description, String constraint, boolean isCaseSensitive) {
		float titleFrequency = getHitFrequency(keywords, title, constraint, isCaseSensitive, HIT_FREQEUNCY_TITLE);
		float descFrequency = getHitFrequency(keywords, description, constraint, isCaseSensitive, HIT_FREQEUNCY_DESCRIPTION);

		return titleFrequency + descFrequency;
	}

	/**
	 * Calculates the hit frequency. Is a wrapper for calculateFrequency. See
	 * calculateFrequency(String keyword, String txt)
	 * 
	 * @param keywords
	 *            array of String words to search for
	 * @param txt
	 *            the String text to search in
	 * @param constraint
	 *            the keyword constraint determines how the keywords are
	 *            searched for
	 * @param isCaseSensitive
	 *            the case sensitivity determines whether case is preserved in
	 *            the searching
	 * @param weight
	 *            the hit frequency to apply
	 * 
	 * @return float freq
	 */
	private static float getHitFrequency(String[] keywords, String txt, String constraint, boolean isCaseSensitive, float weight) {
		float freq = 0f;
		String keyword;
		// Normalize space
		txt = txt.replaceAll("\\s+", " ");
		if (!isCaseSensitive) {
			txt = txt.toLowerCase();
		}
		if (constraint.equals("phrase")) {
			StringBuffer composite = new StringBuffer();
			for (int i = 0; i < keywords.length; i++) {
				keyword = keywords[i];
				if (!isCaseSensitive) {
					keyword = keyword.toLowerCase();
				}
				composite.append(keyword);
				composite.append(" ");
			}
			freq = calculateFrequency(composite.toString().trim(), txt, weight);
		} else {
			for (int i = 0; i < keywords.length; i++) {
				keyword = keywords[i];
				if (!isCaseSensitive) {
					keyword = keyword.toLowerCase();
				}
				freq += calculateFrequency(keyword, txt, weight);
			}
		}
		return freq;
	} // getHitFrequency

	/**
	 * calculateFrequency - calculate the hit frequency for the keyword in the
	 * txt. Frequency is calculated as follows -
	 * 
	 * For each time the keyword matches a group of characters in txt, a value
	 * of 1.0f is added to the frequency.
	 * 
	 * @param keyword
	 *            the String keyword to search for
	 * @param txt
	 *            the String text to search in
	 * @param weight
	 *            the hit frequency to use
	 * 
	 * @return float freq the count, effectively.
	 */
	private static float calculateFrequency(String keyword, String txt, float weight) {
		float freq = 0f;
		int index = txt.indexOf(keyword);
		if (index >= 0) {
			freq += weight;
			while ((index = txt.indexOf(keyword, (index += keyword.length()))) > 0) {
				freq += weight;
			}
		}
		return freq;
	} // calculateFrequency

	/**
	 * staticRelevance - calculate the static relevance of this result using the
	 * calculated hit frequency, highest hit frequency and currentRelevance. The
	 * value is then normalised to the UPPER and LOWER limits.
	 * 
	 * @param currentRelevance
	 *            the current relevance of the result (could be 0)
	 * @param hitFrequency
	 *            the number of occurances of the search terms in this result
	 *            (weighted)
	 * @param highestHitFrequency
	 *            the highest hitFrequency in the result set this result belongs
	 *            to.
	 * 
	 * @return float calculatedRelevance
	 */
	public static float staticRelevance(float currentRelevance, float hitFrequency, float highestHitFrequency) {
		float calculatedRelevance = hitFrequency / highestHitFrequency; // quite
																		// simply
																		// the
																		// ratio
																		// between
																		// this
																		// hit
																		// frequency
																		// and
																		// the
																		// highest

		if (calculatedRelevance > RELEVANCE_UPPER_LIMIT) {
			calculatedRelevance = RELEVANCE_UPPER_LIMIT; // enforce lower
															// limit
		} else if (calculatedRelevance < RELEVANCE_LOWER_LIMIT) {
			calculatedRelevance = RELEVANCE_LOWER_LIMIT; // enforce upper
															// limit
		}
		if (currentRelevance > 0f) {
			calculatedRelevance = (calculatedRelevance + currentRelevance) / 2f; // calculate
																					// the
																					// average
																					// of
																					// the
																					// two
																					// relevances
		}
		return calculatedRelevance;
	} // staticRelevance

	/**
	 * normaliseRelevance - use the relevanceMulitiplier and relevanceOffset
	 * values as specified in the adapter's properties to modify the relevance.
	 * This is used for PRE-PROCESSING the result's relevance before
	 * static/dynamic/any relevance modifiers are applied.
	 * 
	 * @param relevance
	 *            the current relevance of the result (could be 0)
	 * @param relevanceMultiplier
	 *            the multiplier to apply to this relevance FIRST
	 * @param relevanceOffset
	 *            the addition to the result after the relevanceMultiplier has
	 *            been applied
	 * 
	 * @return float newRelevance
	 */
	public static float normaliseRelevance(float relevance, float relevanceMultiplier, float relevanceOffset) {
		float newRelevance = 0f;
		newRelevance = (relevance * relevanceMultiplier) + relevanceOffset;
		if (newRelevance > 1f) {
			newRelevance = 1f;
		} else if (newRelevance < 0f) {
			newRelevance = 0f;
		}
		return newRelevance;
	} // normaliseRelevance

	/**
	 * Maps results set to a Zipfian Distribution, whereby result set ranking is
	 * used to distribute relevance scores
	 * 
	 * @param results
	 *            the List of SearchResults
	 * @param criteria
	 *            the criteria for the search
	 */
	public static void normalizeResultsToZipfianDistribution(List results, SearchCriteria criteria) {
		// TODO: it's possible that sorting by number of matched terms, etc, may
		// improve this
		// particularly when the original ranking from the target collection is
		// not by
		// relevance. We could do that here, or make the adapter/contentHandler
		// do it.
		int resultSetSize = results.size();
		for (int pos = 0; pos < results.size(); pos++) {
			SearchResult result = (SearchResult) results.get(pos);
			result.setRelevance(calculateZipfianRelevanceScore(pos, resultSetSize));
		}
	}

	/**
	 * Returns a score mapped to a (kind of) Zipfian distribution
	 * 
	 * @param pos
	 *            the rank of the result
	 * @param resultSetSize
	 *            the total number of results in the result set
	 * @return a score mapped to a Zipfian distribution
	 */
	public static float calculateZipfianRelevanceScore(int pos, int resultSetSize) {
		// we want rank 1 to be 100%, last to be 10%
		return 1.0F - (0.9F * (float) pos / (float) resultSetSize);
	}

	/** Class name display */
	public final static String classStr = "o.m.d.w.ResultSetUnifier";

} // class ResultSetUnifier

// -- EOF

